"""
Case Type   : cm_ctl可靠性
Case Name   : 双primary主DN仲裁 故障测试-kill -9 cm_agent进程
Create At   : 2022-07-18
Owner       : opentestcase012
Description :
    1.主节点执行事务
    2.备节点指定-M参数为primary,gs_ctl重启并查看集群状态
    3.旧主节点和新主节点注入故障-kill -9 cm_agent进程
    4.几分钟后查看集群状态
    5.备节点再次执行restart并查看集群状态
    6.几分钟后查看集群状态
Expect      :
    1.主节点刷新事务成功
    2.重启成功,集群状态异常,存在双主
    3.注入成功
    4.仲裁成功,只有一个主节点
    5.重启成功,集群状态异常,存在双主
    6.仲裁成功,只有一个主节点
History     :
"""

import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro

Primary_SH = CommonSH('PrimaryDbUser')


@unittest.skipIf(3 != Primary_SH.get_node_num(), '需1主2备环境，否则不执行')
class DefinedResource(unittest.TestCase):
    def setUp(self):
        self.log = Logger()
        self.log.info(f'-----{os.path.basename(__file__)} start-----')
        self.pri_user = Node('PrimaryDbUser')
        self.pri_root = Node('PrimaryRoot')
        self.sta1_user = Node('Standby1DbUser')
        self.sta1_root = Node('Standby1Root')
        self.sta2_user = Node('Standby2DbUser')
        self.constant = Constant()
        self.common = Common()
        self.sh_primary = CommonSH('PrimaryDbUser')
        self.sh_stanby = CommonSH('Standby1DbUser')
        self.sh_file = os.path.join(macro.CMA_DIR_PATH, "cm_test.sh")
        self.sh_logfile = os.path.join(macro.CMA_DIR_PATH, "cm_test.log")

    def test_cm(self):
        text = '----拷贝修改cm_test.sh文件; expect:成功----'
        self.log.info(text)
        self.common.scp_file(self.pri_user, 'cm_test.sh', macro.CMA_DIR_PATH)
        scp_cmd = f"chmod +x {self.sh_file};" \
            f"sed -i 's#env_path#{macro.DB_ENV_PATH}#g' {self.sh_file};" \
            f"sed -i 's#port#{self.pri_user.db_port}#g' {self.sh_file};" \
            f"sed -i 's#database#{self.pri_user.db_name}#g' {self.sh_file};"
        self.log.info(scp_cmd)
        scp_msg = self.common.get_sh_result(self.pri_user, scp_cmd)
        self.assertNotIn('failed', scp_msg, '执行失败:' + text)
        cat_cmd = f"cat {self.sh_file};"
        scp_msg = self.common.get_sh_result(self.pri_user, cat_cmd)
        self.assertIn(macro.DB_ENV_PATH, scp_msg, '执行失败:' + text)
        self.assertIn(self.pri_user.db_port, scp_msg, '执行失败:' + text)

        text = '----step1:主节点执行事务; expect:主节点刷新事务成功----'
        self.log.info(text)
        sh_cmd = f'sh {self.sh_file} &>{self.sh_logfile} &'
        self.log.info(sh_cmd)
        sh_thread = ComThread(self.pri_user.sh, args=(sh_cmd,))
        sh_thread.setDaemon(True)
        sh_thread.start()
        sh_thread.join(120)

        text = '----查看主节点事务执行结果; expect:主节点刷新事务成功----'
        self.log.info(text)
        time.sleep(2)
        cat_cmd = f'cat {self.sh_logfile}| tail -n 50;'
        self.log.info(cat_cmd)
        cat_msg = self.common.get_sh_result(self.pri_user, cat_cmd)
        for err_msg in self.constant.SQL_WRONG_MSG:
            self.assertNotIn(err_msg, cat_msg)
        self.assertIn(self.constant.TABLE_DROP_SUCCESS, cat_msg,
                      "执行失败:" + text)

        text = '----step2:备节点指定-M参数为primary,gs_ctl重启并查看集群状态; ' \
               'expect:重启成功,集群状态异常,存在双主----'
        self.log.info(text)
        restart_cmd = f'source {macro.DB_ENV_PATH};' \
                      f'cm_ctl pause;' \
                      f'gs_ctl restart -D {macro.DB_INSTANCE_PATH} -M primary;' \
                      f'cm_ctl query -Cv;'
        self.log.info(restart_cmd)
        restart_msg = self.common.get_sh_result(self.sta1_user, restart_cmd)
        self.assertIn('The cluster has been paused.', restart_msg,
                      "执行失败:" + text)
        self.assertIn(self.constant.RESTART_SUCCESS_MSG, restart_msg,
                      "执行失败:" + text)
        self.assertNotIn(self.constant.CLUSTER_NORMAL_MSG, restart_msg,
                         "执行失败:" + text)
        self.assertIn('P Primary', restart_msg, "执行失败:" + text)
        self.assertTrue('S Primary' in restart_msg or
                        'S Down    Starting' in restart_msg, '执行失败:' + text)

        try:
            threads = []
            text = '----step3:旧主节点和新主节点注入故障-kill -9 cm_agent进程; ' \
                   'expect:注入故障成功----'
            self.log.info(text)
            kill_cmd = '''ps ux|grep "cm_agent"|grep -v grep|\
            tr -s ' '|cut -d ' ' -f 2|xargs kill -9;'''
            kill_thread = ComThread(self.sta1_user.sh, args=(kill_cmd,))
            threads.append(kill_thread)
            kill_thread = ComThread(self.pri_user.sh, args=(kill_cmd,))
            threads.append(kill_thread)

            text = '----启动线程获取结果----'
            self.log.info(text)
            thread_results = []
            for t in threads:
                t.setDaemon(True)
                t.start()
            for t in threads:
                t.join(60)
                thread_results.append(t.get_result())
                self.log.info(t.get_result())
            self.assertEquals("", thread_results[0], '执行失败:' + text)
            self.assertEquals("", thread_results[1], '执行失败:' + text)
        finally:
            time.sleep(500)

        text = '----step4:查看集群状态; expect:仲裁成功,只有一个主节点----'
        self.log.info(text)
        show_status = f'source {macro.DB_ENV_PATH};' \
                      f'cm_ctl resume;' \
                      f'sleep 60;' \
                      f'cm_ctl query -i -v -C -d'
        self.log.info(show_status)
        status_msg = self.common.get_sh_result(self.pri_user, show_status)
        self.assertIn('The cluster has been resumed.', status_msg,
                      "执行失败:" + text)
        self.assertIn(self.constant.CLUSTER_NORMAL_MSG, status_msg,
                      "执行失败:" + text)
        self.assertEqual(1, status_msg.count('Primary Normal'), "执行失败:" + text)

        text = '----恢复主备初始状态; expect:成功----'
        self.log.info(text)
        switchover_cmd = f'source {macro.DB_ENV_PATH};' \
            f'cm_ctl switchover -a;'
        status_msg = self.common.get_sh_result(self.pri_user, switchover_cmd)
        self.assertIn(self.constant.cm_switchover_success_msg, status_msg,
                      "执行失败:" + text)
        time.sleep(2)

        text = '----step5:备节点再次执行restart并查看集群状态; expect:重启成功,集群状态异常,存在双主----'
        self.log.info(text)
        self.log.info(restart_cmd)
        restart_msg = self.common.get_sh_result(self.sta1_user, restart_cmd)
        self.assertIn(self.constant.RESTART_SUCCESS_MSG, restart_msg,
                      "执行失败:" + text)
        self.assertNotIn(self.constant.CLUSTER_NORMAL_MSG, restart_msg,
                         "执行失败:" + text)
        self.assertIn('P Primary', restart_msg, "执行失败:" + text)
        self.assertTrue('S Primary' in restart_msg or
                        'S Down    Starting' in restart_msg, '执行失败:' + text)

        text = '----step6:查看集群状态; expect:仲裁成功,只有一个主节点----'
        self.log.info(text)
        time.sleep(300)
        status_msg = self.common.get_sh_result(self.pri_user, show_status)
        self.assertIn(self.constant.CLUSTER_NORMAL_MSG, status_msg,
                      "执行失败:" + text)
        self.assertEqual(1, status_msg.count('Primary Normal'), "执行失败:" + text)

    def tearDown(self):
        self.log.info('-----清理环境-----')
        text1 = '-----删除sh文件; expect:删除成功-----'
        self.log.info(text1)
        kill_cmd = f"ps ux|grep \"{self.sh_file}\"|grep -v grep|" \
            f"tr -s ' '|cut -d ' ' -f 2|xargs kill -9;"
        self.log.info(kill_cmd)
        self.common.get_sh_result(self.pri_user, kill_cmd)
        rm_cmd = f'rm -rf {self.sh_logfile} {self.sh_file};'
        self.log.info(rm_cmd)
        self.common.get_sh_result(self.pri_root, rm_cmd)
        check_cmd = f'if [[ -e {self.sh_logfile} && -e {self.sh_file} ]]; ' \
            f'then echo "exists"; else echo "not exists"; fi'
        rm_msg_pri = self.common.get_sh_result(self.pri_root, check_cmd)

        text2 = '-----主备恢复为初始状态; expect:成功-----'
        self.log.info(text2)
        clear_cmd = f'source {macro.DB_ENV_PATH};' \
                    f'cm_ctl resume;' \
                    f'cm_ctl start;' \
                    f'cm_ctl switchover -a;' \
                    f'sleep 5;' \
                    f'cm_ctl query -Cv;'
        self.log.info(clear_cmd)
        clear_msg = self.common.get_sh_result(self.pri_user, clear_cmd)

        text5 = '----数据恢复为初始状态; expect:成功----'
        del_cmd = "drop table t_cm_001 cascade;"
        del_res = self.sh_primary.execut_db_sql(del_cmd)
        self.log.info(del_res)
        self.assertIn(del_res, ('ERROR:  table "t_cm_001" does not exist', 'DROP TABLE'), "执行失败:" + text5)

        self.assertEqual('not exists', rm_msg_pri, '执行失败:' + text1)
        self.assertIn('The cluster has been resumed.', clear_msg,
                      "执行失败:" + text2)
        self.assertIn(self.constant.cm_start_success_msg, clear_msg,
                      "执行失败:" + text2)
        self.assertIn(self.constant.cm_switchover_success_msg, clear_msg,
                      "执行失败:" + text2)
        self.assertIn(self.constant.CLUSTER_NORMAL_MSG, clear_msg,
                      "执行失败:" + text2)
        self.assertIn('P Primary Normal', clear_msg, "执行失败:" + text2)
        self.assertIn('S Standby Normal', clear_msg, "执行失败:" + text2)
        self.log.info(f'-----{os.path.basename(__file__)} end-----')

